package com.frontalcode 
{
	/**
	 * @private
	 */
    internal class ActionScriptTokenizer extends Tokenizer
	{
		function ActionScriptTokenizer ( tokens : Array, options : Object = null )
		{
			super ( tokens, options );
			
			setOptionsFromLoaderParameters ( );
		}
	
		/*
			Function: getNextToken
			
				Return the next token found in the raw character stream.
			
			Parameters:
			
				codeStream - See internalEvaluate ( ) for structure of codeStream parameter.
			
			Returns:
				
				The token descriptor.
		*/
		internal function getNextToken ( codeStream : Object ) : Object
		{
			// Skip whitespace.
			//
			
			var nextChar : String;
			var hasPrecedingLineTerminator : Boolean = codeStream.index == 0 ? true : false;
			
			while ( ( nextChar = codeStream.code.charAt ( codeStream.index ) ) == ' ' || nextChar == '\t' || nextChar == '\r' || nextChar == '\n' )
			{
				if ( nextChar == '\r' || nextChar == '\n' ) hasPrecedingLineTerminator = true;
				codeStream.index++;
			}
				
			if ( codeStream.index >= codeStream.code.length )
			{
				return undefined;
			}

			var tokenDefn : Object;
			
			// Find a matching, self-delimiting token.
			//
			
			if ( selfDelimitingTokenCheck.indexOf ( codeStream.code.charAt ( codeStream.index ) ) >= 0 )
			{
				var scanString : String = codeStream.code.substr ( codeStream.index, allTokens [ 0 ].token.length );
				
				while ( scanString.length > 0 )
				{
					if ( allSelfDelimitingTokensHash [ scanString ] != undefined )
					{
						tokenDefn = copyObject ( allSelfDelimitingTokensHash [ scanString ] );
						break;
					}
					
					scanString = scanString.substr ( 0, scanString.length - 1 );
				}
			}
			
			if ( tokenDefn == null || tokenDefn.token == "." )
			{
				var scan : Number = codeStream.index + ( tokenDefn == null ? 0 : 1 );
				nextChar = codeStream.code.charAt ( scan ).toLowerCase ( );
				
				if ( nextChar >= "0" && nextChar <= "9" )
				{
					// It is a numeric literal.
					//
				
					var completeNumber : String = ( tokenDefn == null ? "" : "." ) + nextChar;
					
					while ( ( nextChar = codeStream.code.charAt ( ++scan ).toLowerCase ( ) ) >= "0" && nextChar <= "9" ) completeNumber += nextChar;
					
					if ( completeNumber == "0" && nextChar == "x" )
					{
						// Parse a hex number.
						//
						
						completeNumber += "x";
						
						while ( ( ( nextChar = codeStream.code.charAt ( ++scan ).toLowerCase ( ) ) >= "0" && nextChar <= "9" ) || ( nextChar >= "a" && nextChar <= "f" ) ) completeNumber += nextChar;
						
						if ( completeNumber == "0x" )
						{
							throw new FrontalError ( "Illegal number \"" + completeNumber + "\" at " + getCodeSnippet ( codeStream.code, undefined, codeStream.index ) );
						}
					}
					else
					{
						if ( nextChar == "." && tokenDefn == null )
						{
							completeNumber += ".";
						
							while ( ( nextChar = codeStream.code.charAt ( ++scan ).toLowerCase ( ) ) >= "0" && nextChar <= "9" ) completeNumber += nextChar;
						}
						
						if ( nextChar == "e" )
						{
							completeNumber += "e";
							
							if ( ( nextChar = codeStream.code.charAt ( ++scan ).toLowerCase ( ) ) == "+" || nextChar == "-" || ( nextChar >= "0" && nextChar <= "9" ) )
							{
								completeNumber += nextChar;
							}
							
							while ( ( nextChar = codeStream.code.charAt ( ++scan ).toLowerCase ( ) ) >= "0" && nextChar <= "9" ) completeNumber += nextChar;
						}
					}
					
					// Extract the token from the code stream.
					//
					
					completeNumber = codeStream.code.substr ( codeStream.index, completeNumber.length );
					
					if ( isNaN ( Number ( completeNumber ) ) )
					{
						throw new FrontalError ( "Illegal number \"" + completeNumber + "\" at " + getCodeSnippet ( codeStream.code, undefined, codeStream.index ) );
					}
					
					tokenDefn = {
							value: completeNumber,
							token: "Number",
							precedence: 0
						};
				}
				else if ( tokenDefn == null || tokenDefn.token == undefined )
				{
					// It is an identifier.
					//
					
					if ( nextChar != "_" && ( nextChar < "a" || nextChar > "z" ) )
					{
						throw new FrontalError ( "Identifier cannot start with \"" + nextChar + "\" at " + getCodeSnippet ( codeStream.code, undefined, codeStream.index ) );
					}
					
					while ( ( ( nextChar = codeStream.code.charAt ( ++scan ).toLowerCase ( ) ) >= "0" && nextChar <= "9" ) || nextChar == "_" || ( nextChar >= "a" && nextChar <= "z" ) ) { }
	
					var identifier : String = codeStream.code.substring ( codeStream.index, scan );
					
					// Check if it's a token. 
					//
					
					if ( allTokensHash.hasOwnProperty ( identifier ) )
					{
						tokenDefn = copyObject ( allTokensHash [ identifier ] );
					}
					else
					{
						tokenDefn = {
								value: identifier,
								token: "Identifier", 
								precedence: 0
							};
					}
				}
				
				// Else it's just a dot.
				//
			}
			
			tokenDefn.codeIndex = codeStream.index;
			tokenDefn.hasPrecedingLineTerminator = hasPrecedingLineTerminator;
	 
			if ( tokenDefn.value != undefined )
			{
				codeStream.index += tokenDefn.value.length;
			}
			else
			{
				codeStream.index += tokenDefn.token.length;
			}
			
			return tokenDefn;
		}
	
		/*
			Function: tokenizeCode
			
				Convert a code string into a stream of tokens.
				
				We also cleanup the token stream by converting quoted-literals into
				String tokens and by removing comments.
				
			Returns:
			
				The tokenized code stream.
		*/
		override internal function tokenizeCode ( code : String, debug : * ) : Object
		{
			if ( debug != null ) this.debug = Boolean ( debug );
			
			var codeStream : Object = { code: code, index: 0 };
			var tokenStream : Array = new Array ( );
			
			while ( codeStream.index < codeStream.code.length )
			{
				var tokenDefn : Object = getNextToken ( codeStream );
				
				if ( tokenDefn == null )
				{
					// Created by trailing spaces.
					//
					
					break;
				}
				
				// Do some special processing on comments and quoted strings.
				//
				
				var discardToken : Boolean = false;
				
				switch ( tokenDefn.token )
				{
					case "//":
						// Jump to the next line.
						//
						var nextCR : Number = codeStream.code.indexOf ( "\r", codeStream.index );
						if ( nextCR < 0 ) nextCR = codeStream.code.length;
						var nextNL : Number = codeStream.code.indexOf ( "\n", codeStream.index );
						if ( nextNL < 0 ) nextNL = codeStream.code.length;
						codeStream.index = Math.min ( nextCR, nextNL );
						while ( codeStream.index < codeStream.code.length && "\r\n".indexOf ( codeStream.code.charAt ( codeStream.index ) ) == 0 ) codeStream.index++;
						discardToken = true;
						break;
						
					case "/*":
						// Jump to the close comment tag.
						//
				
						var closeComment : Number = codeStream.code.indexOf ( "*/", codeStream.index );
						
						if ( closeComment >= 0 )
						{
							codeStream.index = closeComment + 2;
						}
						
						discardToken = true;
						break;
				
					case "\"":
					case "'":
						// We want to find the next unescaped quote and use the
						// skipped characters as a String.
						//
						
						var nextQuote : Number = codeStream.code.indexOf ( tokenDefn.token, codeStream.index );
						
						while ( nextQuote > 0 && codeStream.code.charAt ( nextQuote - 1 ) == "\\" && codeStream.code.charAt ( nextQuote - 2 ) != "\\" )
						{
							nextQuote = codeStream.code.indexOf ( tokenDefn.token, nextQuote + 1 );
						}
						
						if ( nextQuote < 0 )
						{
							throw new FrontalError ( "Unmatched quote: " + codeStream.code.substr ( codeStream.index - 1 ) + " at " + getCodeSnippet ( codeStream.code, undefined, codeStream.index ) );
						}
						
						var delimited : String = codeStream.code.substring ( codeStream.index, nextQuote );
						var stringLiteral : String = "";
						
						// Unescape any escapes.
						//
						
						var scan : Number = 0;
						
						while ( scan < delimited.length )
						{
							var nextChar : String = delimited.charAt ( scan );
							
							if ( nextChar == "\\" )
							{
								var processedChars : uint = 2;
								
								if ( scan + 1 >= delimited.length ) throw new FrontalError ( "Incomplete escape in string: " + delimited + " at " + getCodeSnippet ( codeStream.code, undefined, codeStream.index ) );
								
								switch  ( delimited.charAt ( scan + 1 ) )
								{
									case "\\":
										stringLiteral += "\\";
										break;
										
									case "/":
										// Like JavaScript.
										stringLiteral += "/";
										break;
										
									case "b":
										stringLiteral += "\b";
										break;
										
									case "t":
										stringLiteral += "\t";
										break;
										
									case "n":
										stringLiteral += "\n";
										break;
										
									case "v":
										stringLiteral += "\v";
										break;
										
									case "f":
										stringLiteral += "\f";
										break;
										
									case "r":
										stringLiteral += "\r";
										break;
										
									case "'":
										stringLiteral += "'";
										break;
										
									case "\"":
										stringLiteral += "\"";
										break;
										
									case "u":
										processedChars = 6;
										if ( scan + 5 >= delimited.length ) throw new FrontalError ( "Incomplete escape in string: " + delimited + " at " + getCodeSnippet ( codeStream.code, undefined, codeStream.index ) );
										stringLiteral += String.fromCharCode ( Number ( "0x" + delimited.substr ( scan + 2, 4 ) ) );
										break;
										
									case "x":
										processedChars = 4;
										if ( scan + 5 >= delimited.length ) throw new FrontalError ( "Incomplete escape in string: " + delimited + " at " + getCodeSnippet ( codeStream.code, undefined, codeStream.index ) );
										stringLiteral += unescape ( "%" + delimited.substr ( scan + 2, 2 ) );
										break;
										
									default:
										throw new FrontalError ( "Unknown escape code in string: " + delimited + " at " + getCodeSnippet ( codeStream.code, undefined, codeStream.index ) );
										break;
								}
								
								scan += processedChars;
							}
							else
							{
								stringLiteral += nextChar;
								scan++;
							}
						}
						
						// Remove the quote token.
						//
	
						discardToken = true;
						
						// Add the StringLiteral token.
						//
	
						tokenStream.push ( { value: stringLiteral, token: "String", precedence: 0, codeIndex: codeStream.index } );
			
						if ( debug ) trace ( "add token " + Debugger.objectToString ( tokenStream [ tokenStream.length - 1 ], 0, 2, false ) );
						
						// Remove the closing quote.
						//
						
						codeStream.index = nextQuote + 1;
						
						break;
				}
				
				if ( ! discardToken )
				{
					tokenStream.push ( tokenDefn );
			
					if ( debug ) trace ( "add token " + Debugger.objectToString ( tokenStream [ tokenStream.length - 1 ], 0, 2, false ) );
				}
			}
			
			return { tokens: tokenStream, index: 0 };
		}
	}
}
